import html
from html.parser import HTMLParser
import pandas as pd
from w3lib.html import (remove_tags, 
                        remove_comments, 
                        remove_tags_with_content,
                        replace_escape_chars,
                        replace_entities)
from urllib.parse import urljoin
from parsel import Selector

class WebCleaner(HTMLParser):
    def __init__(self):
        super().__init__()
        self.reset()
        self.fed = []
    
    def handle_data(self, d):
        self.fed.append(d)
    
    def get_data(self):
        return ' '.join(self.fed)
    
    def parse_html(self, html_content):
        if not html_content:
            return ""
        else:
            self.feed(html.unescape(html_content))
            text = self.get_data()
            return text
    
    def clean_html(self, html_content):
        text = remove_tags_with_content(html_content, which_ones=('script', 'style', 'noscript', 'header', 'footer', 'svg', 'img', 'figure', 'figcaption', 'meta', 'link', 'iframe'))
        text = remove_comments(text)
        text = remove_tags(text)
        text = replace_entities(text)
        text = replace_escape_chars(text, which_ones=('\n', '\t', '\r'), replace_by=' ')
        text = ' '.join(text.split())
        return text

    def extract_links(self, html_content):
        selector = Selector(text=html_content)
        links = selector.css('a::attr(href)').getall()
        return links
    